package com.ruoyi.house.crawling;
 
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.sql.SQLException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
 
public class CrawlingDoubanHouseInfo {
	public static String DOU_BAN_URL = "https://www.douban.com/group/longgangzufang/discussion?start={pageStart}";
	public static void main(String[] args) throws IOException, ClassNotFoundException, SQLException, InterruptedException {
		
//		Class.forName("com.mysql.jdbc.Driver");
//		String sqlUrl = "jdbc:mysql://localhost:3306/douban?characterEncoding=UTF-8";
//		Connection conn = DriverManager.getConnection(sqlUrl, "root", "VisionKi");
//		Statement stat = conn.createStatement();
		
		int pageStrat = 0;
		while(true) {
			try {
				URL url = new URL(DOU_BAN_URL.replace("{pageStart}",pageStrat+""));
				System.out.println("当前页面：" + DOU_BAN_URL.replace("{pageStart}",pageStrat+""));
				HttpURLConnection connection = (HttpURLConnection)url.openConnection();
	            //设置请求方式
	            connection.setRequestMethod("GET");
	            // 10秒超时
	            connection.setConnectTimeout(10000);
	            connection.setReadTimeout(10000);
	            
	            connection.setRequestProperty("Cookie", "这里按照前面说的登陆后F12拿到cookie放在这里");
	            connection.setRequestProperty("Host", "www.douban.com");
	            connection.setRequestProperty("Referer", "https://www.douban.com/group/longgangzufang/discussion?start=25");
	            connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36");
	            
	            //连接
	            connection.connect();
 
 
	            //得到响应码
	            int responseCode = connection.getResponseCode();
 
	            if(responseCode == HttpURLConnection.HTTP_OK){
	                //得到响应流
	                InputStream inputStream = connection.getInputStream();
	                //获取响应
	                BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
	                String returnStr = "";
	                String line;
	                while ((line = reader.readLine()) != null){
	                    returnStr+=line + "\r\n";
	                }
	                //该干的都干完了,记得把连接断了
	                reader.close();
	                inputStream.close();
	                connection.disconnect();
//	                System.out.println(returnStr);
	                Pattern p = Pattern.compile("<a href=\"([^\"]*)\" title=\"[^\"]*\" class=\"\">[^\"]*</a>");
	        	    Matcher m = p.matcher(returnStr); 
	        	    while(m.find()) {
	            	    Thread.sleep(500);
	        	    	try {
		        	    	String tempUrlStr = m.group(1);
		        	    	System.out.println("		当前链接：" + tempUrlStr);
		        	    	URL tempUrl = new URL(tempUrlStr);
		        			HttpURLConnection tempConnection = (HttpURLConnection)tempUrl.openConnection();
		                    //设置请求方式
		        			tempConnection.setRequestMethod("GET");
		        			// 10秒超时
		        			tempConnection.setConnectTimeout(10000);
		        			tempConnection.setReadTimeout(10000);
//		        			tempConnection.setRequestProperty("Cookie", "这里按照前面说的登陆后F12拿到cookie放在这里");
		        			tempConnection.setRequestProperty("Cookie", "bid=I1VqxJNr97M; _pk_id.100001.8cb4=9415e08ad7b3161b.1718813888.; _pk_ses.100001.8cb4=1; ap_v=0,6.0; __utma=30149280.212401624.1718813890.1718813890.1718813890.1; __utmc=30149280; __utmz=30149280.1718813890.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmt=1; douban-fav-remind=1; __yadk_uid=1nrYm6GW70nLKujmxGM7sPwb4g4p88Dw; __utmb=30149280.18.5.1718813943057");
		        			tempConnection.setRequestProperty("Host", "www.douban.com");
		        			tempConnection.setRequestProperty("Referer", "https://www.douban.com/group/longgangzufang/discussion?start=25");
		        			tempConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36");
		                    //连接
		        			tempConnection.connect();
		                    //得到响应码
		                    int tempResponseCode = tempConnection.getResponseCode();
 
		                    if(tempResponseCode == HttpURLConnection.HTTP_OK){
		                        //得到响应流
		                        InputStream tempInputStream = tempConnection.getInputStream();
		                        //获取响应
		                        BufferedReader tempReader = new BufferedReader(new InputStreamReader(tempInputStream));
		                        String tempReturnStr = "";
		                        String tempLine;
		                        while ((tempLine = tempReader.readLine()) != null){
		                        	tempReturnStr += tempLine + "\r\n";
		                        }
		                        
 
		                        Pattern p2 = Pattern.compile("\"text\": \"([^\"]*)\",\r\n" + 
		                        		"	\"name\": \"([^\"]*)\",\r\n" + 
		                        		"	\"url\": \"([^\"]*)\",\r\n" + 
		                        		"  \"commentCount\": \"[^\"]*\",\r\n" + 
		                        		"  \"dateCreated\": \"([^\"]*)\",");
		                	    Matcher m2 = p2.matcher(tempReturnStr); 
		                	    while(m2.find()) {
		                	    	System.out.println(m2.group(1));
		                	    	System.out.println(m2.group(2));
		                	    	System.out.println(m2.group(3));
		                	    	System.out.println(m2.group(4));
//		                	    	stat.executeUpdate("INSERT INTO house(title,content,time,house_url) VALUES ('" + m2.group(2).replaceAll("[\\x{10000}-\\x{10FFFF}]", "") + "','" + m2.group(1).replaceAll("[\\x{10000}-\\x{10FFFF}]", "") + "','" + m2.group(4).replace("T"," ") + "','" + m2.group(3) + "');");
		                	    }
			                    tempReader.close();
			                    tempInputStream.close();
			                    tempConnection.disconnect();
		                    }
	        	    		
	        	    	}catch(Exception e) {
	        	    		e.printStackTrace();
	        	    	}
	        	    }
	        	    System.out.println("换页");
	            }
			}catch(Exception e) {
				e.printStackTrace();
			}
    	    pageStrat+=25;
		}	
	}
}